import itertools
from collections import deque
import pathlib
import logging

from os import unlink

import unicodedata
from jinja2 import Template
from pygments.lexers import q

from typing import List, Union
# 输入目录 扫描输入路径下的所有文件
out_path_sx = '山西数据'
out_path_nm = '内蒙数据'
out_path_jl = '吉林数据'
out_path_hl = '黑龙江数据'
out_path_e = 'EXCEPTION'

ACTVHH_LEN = 32
RETTXN_LEN = 37
CDTDLG_LEN = 33

# 机构号下表索引
ACTVHH_INDEX = 1
RETTXN_INDEX = 13
RETTXN_INDEX_B = 15
CDTDLG_INDEX = 20
CDTDLG_INDEX_B = 21

# 山西机构号范围
sx_org_begin = 1006
sx_org_end = 1299
# 内蒙机构号范围
nm_org_begin = 1301
nm_org_end = 1599
# 吉林机构号范围
jl_org_begin = 2009
jl_org_end = 2299
# 黑龙江机构号范围
hl_org_begin = 2300
hl_org_end = 2557

# 文件开头/结尾标识
begin= '|||BEGIN|||\n'
end = '|||END|||\n'

# 配置文件


""" 配置文件通用参数 ============================================================== """
# 需要统计条目数
user = "SBS3"
recs_amt   = 0
sep        = "|"
# 统计各个文件数目
part_info  = 0
sys_name   = "SBS"
sys_ver    = "V03.01"
sys_comm   = "UNIX综合业务系统"
infc_id    = "HB"

# 输出配置文件路径 山西，内蒙 ，吉林 ，黑龙
config_paths = ['']*4
# 备份时间
back_times = ['']*4
# 输出个数
part_infos = [0]*4
# 文件条数
recs_amts = [0]*4
# 计数器
account = 0
"""=========================最终文件输出路径=========================="""
processing_path = ['']*4
"""==========================================="""
remove = True

# actvhh-xml 配置文件参数
actvhh_config = {
    "tab_name" : "SBS3.ACTVHH",
    "tab_comm" : "传票历史文件",
    "cols_amt" : 30,
    "daterange": "ERYDAT"
}

# actvhh-xml 配置文件参数
cdtdlg_config = {
    "tab_name" : "SBS3.CDTDLG",
    "tab_comm" : "卡交易流水文件",
    "cols_amt" : 31,
    "daterange": "SCTDAT"
}
# rettxn-xml 配置文件参数
rettxn_config = {
    # 会变的数据
    "tab_name"    : "SBS3.RETTXN",
    "tab_comm"    : "传票历史文件",
    "cols_amt"    : 35,
    # 设置日期字段
    "daterange"   : "TXNDAT",
}

# actvhh字段列表（按需增删）
actvhh_columns = [
    {"colname": "ORGIDT", "type": "varchar", "length": 4, "colcomm": "机构号"},
    {"colname": "TLRNUM", "type": "varchar", "length": 4, "colcomm": "柜员号"},
    {"colname": "VCHSET", "type": "number",  "length": 4, "colcomm": "输入当天套顺序号"},
    {"colname": "SETSEQ", "type": "number",  "length": 2, "colcomm": "输入当天套内顺序号"},
    {"colname": "ERYDAT", "type": "date",    "length": 8, "colcomm": "记帐日"},
    {"colname": "OURREF", "type": "varchar",    "length": 16, "colcomm": "业务引用号"},
    {"colname": "ORGID3", "type": "varchar",    "length": 4, "colcomm": "账户行机构号"},
{"colname": "CUSIDT", "type": "varchar",    "length": 7, "colcomm": "客户号"},
{"colname": "APCODE", "type": "varchar",    "length": 4, "colcomm": "核算码"},
{"colname": "CURCDE", "type": "varchar",    "length": 3, "colcomm": "货币码"},
{"colname": "TXNAMT", "type": "number",    "length": 17, "colcomm": "交易金额"},
{"colname": "BOKBAL", "type": "number",    "length": 17, "colcomm": "交易后账面余额"},
{"colname": "VALDAT", "type": "date",    "length": 8, "colcomm": "起息日"},
{"colname": "ANACDE", "type": "varchar",    "length": 4, "colcomm": "分析码/统计号"},
{"colname": "FURINF", "type": "varchar",    "length": 32, "colcomm": "摘要"},
{"colname": "FXRATE", "type": "number",    "length": 12, "colcomm": "汇率"},
{"colname": "VCHATT", "type": "number",    "length": 3, "colcomm": "附件数"},
{"colname": "THRREF", "type": "varchar",    "length": 16, "colcomm": "对方业务引用号"},
{"colname": "VCHAUT", "type": "varchar",    "length": 4, "colcomm": "授权柜员号"},
{"colname": "VCHANO", "type": "varchar",    "length": 6, "colcomm": "授权号"},
{"colname": "DEPNUM", "type": "varchar",    "length": 2, "colcomm": "部门号"},
{"colname": "REGNUM", "type": "varchar",    "length": 20, "colcomm": "建卡销卡凭证编号"},
{"colname": "TXNBAK", "type": "varchar",    "length": 7, "colcomm": "交易行号"},
{"colname": "ACTBAK", "type": "varchar",    "length": 7, "colcomm": "账户行号"},
{"colname": "CLRBAK", "type": "varchar",    "length": 7, "colcomm": "清算行号"},
{"colname": "ORGID4", "type": "varchar",    "length": 3, "colcomm": "清算行机构号"},
{"colname": "ERYTYP", "type": "varchar",    "length": 1, "colcomm": "传票输入方式"},
{"colname": "VCHDAT", "type": "date",    "length": 8, "colcomm": "输入日期"},
{"colname": "VCHTIM", "type": "varchar",    "length": 8, "colcomm": "输入时间（char型）"},
{"colname": "RECSTS", "type": "varchar",    "length": 1, "colcomm": "记录状态"}
]

# rettxn字段列表（按需增删）
rettxn_columns = [
    {"colname": "BOKNUM", "type": "varchar", "length": 19, "colcomm": "存款账户号"},
    {"colname": "TXNDAT", "type": "date", "length": 8, "colcomm": "交易日期"},
    {"colname": "CLRNUM", "type": "varchar", "length": 7, "colcomm": "清算结构号"},
    {"colname": "TRNNUM", "type": "varchar", "length": 2, "colcomm": "终端号"},
    {"colname": "TXNSEQ", "type": "varchar", "length": 5, "colcomm": "交易流水号"},
    {"colname": "ACTTYP", "type": "varchar", "length": 2, "colcomm": "账户号类型"},
    {"colname": "IPTACT", "type": "varchar", "length": 22, "colcomm": "账户号"},
    {"colname": "ACTNAM", "type": "varchar", "length": 72, "colcomm": "客户户名"},
    {"colname": "PASSNO", "type": "varchar", "length": 18, "colcomm": "身份证"},
    {"colname": "TXNCDE", "type": "varchar", "length": 4, "colcomm": "交易码"},
    {"colname": "TXNTIM", "type": "varchar", "length": 6, "colcomm": "交易时间"},
    {"colname": "VCHSET", "type": "varchar", "length": 4, "colcomm": "传票号"},
    {"colname": "ACTORG", "type": "varchar", "length": 4, "colcomm": "帐务机构"},
    {"colname": "ACTDEP", "type": "varchar", "length": 2, "colcomm": "帐务部门"},
    {"colname": "AGTIDT", "type": "varchar", "length": 4, "colcomm": "代理结构"},
    {"colname": "AGTDEP", "type": "varchar", "length": 2, "colcomm": "代理部门"},
    {"colname": "TLRNUM", "type": "varchar", "length": 4, "colcomm": "交易柜员"},
    {"colname": "AUTTLR", "type": "varchar", "length": 4, "colcomm": "授权柜员"},
    {"colname": "DRAMNT", "type": "number", "length": 17, "colcomm": "借方发生额"},
    {"colname": "CRAMNT", "type": "number", "length": 17, "colcomm": "贷方发生额"},
    {"colname": "PREBAL", "type": "number", "length": 17, "colcomm": "交易前余额"},
    {"colname": "PSTBAL", "type": "number", "length": 17, "colcomm": "交易后余额"},
    {"colname": "TOTINT", "type": "number", "length": 17, "colcomm": "总利息"},
    {"colname": "COMFEE", "type": "number", "length": 17, "colcomm": "手续费"},
    {"colname": "PAPTYP", "type": "varchar", "length": 1, "colcomm": "支票种类"},
    {"colname": "PAPNUM", "type": "varchar", "length": 10, "colcomm": "支票号"},
    {"colname": "DRAIDT", "type": "varchar", "length": 1, "colcomm": "取款用途"},
    {"colname": "RVSFLG", "type": "varchar", "length": 1, "colcomm": "冲正标志"},
    {"colname": "OPTTYP", "type": "varchar", "length": 1, "colcomm": "操作标志"},
    {"colname": "DUEFLG", "type": "varchar", "length": 1, "colcomm": "强制标志"},
    {"colname": "NBKFLG", "type": "varchar", "length": 1, "colcomm": "无折标志"},
    {"colname": "CSHFLG", "type": "varchar", "length": 1, "colcomm": "现金标志"},
    {"colname": "WATTYP", "type": "varchar", "length": 1, "colcomm": "流水号类型"},
    {"colname": "WATNUM", "type": "varchar", "length": 6, "colcomm": "证券流水号"},
    {"colname": "TXNREF", "type": "varchar", "length": 30, "colcomm": "摘要"}
]

#cdtdlg字段列表（按需增删）
cdtdlg_columns = [
    {"colname": "SCTDAT", "type": "date", "length": 8, "colcomm": "计帐日期"},
    {"colname": "SEQNUM", "type": "number", "length": 8, "colcomm": "交易流水号"},
    {"colname": "TLRNUM", "type": "varchar", "length": 4, "colcomm": "柜员号"},
    {"colname": "TXNCNT", "type": "number", "length": 6, "colcomm": "柜员流水号"},
    {"colname": "TXNDAT", "type": "date", "length": 8, "colcomm": "交易日期"},
    {"colname": "TXNTIM", "type": "varchar", "length": 6, "colcomm": "交易时间"},
    {"colname": "TXNTYP", "type": "varchar", "length": 1, "colcomm": "交易类型"},
    {"colname": "CADNUM", "type": "varchar", "length": 20, "colcomm": "卡号"},
    {"colname": "ACTTYP", "type": "varchar", "length": 2, "colcomm": "帐号类型"},
    {"colname": "IPTACT", "type": "varchar", "length": 22, "colcomm": "帐号"},
    {"colname": "CURCDE", "type": "varchar", "length": 3, "colcomm": "货币"},
    {"colname": "DPTTYP", "type": "varchar", "length": 2, "colcomm": "存款种类"},
    {"colname": "SHUTNM", "type": "varchar", "length": 2, "colcomm": "单号"},
    {"colname": "MCHNUM", "type": "varchar", "length": 15, "colcomm": "商户号"},
    {"colname": "TERMID", "type": "varchar", "length": 8, "colcomm": "终端号"},
    {"colname": "DEVNUM", "type": "varchar", "length": 16, "colcomm": "设备号"},
    {"colname": "TXNAMT", "type": "number", "length": 17, "colcomm": "交易金额"},
    {"colname": "TXNFE1", "type": "number", "length": 10, "colcomm": "发卡行手续费"},
    {"colname": "TXNFE2", "type": "number", "length": 10, "colcomm": "代办行手续费"},
    {"colname": "TXNIBK", "type": "varchar", "length": 4, "colcomm": "发卡行号"},
    {"colname": "TXNABK", "type": "varchar", "length": 4, "colcomm": "代办行号"},
    {"colname": "CADTYP", "type": "varchar", "length": 1, "colcomm": "卡类型"},
    {"colname": "SYSTRC", "type": "varchar", "length": 7, "colcomm": "系统跟踪号"},
    {"colname": "VCHSET", "type": "varchar", "length": 4, "colcomm": "传票套号"},
    {"colname": "REFNU1", "type": "varchar", "length": 12, "colcomm": "系统参考号1"},
    {"colname": "REFNU2", "type": "varchar", "length": 12, "colcomm": "系统参考号2"},
    {"colname": "CCLFLG", "type": "varchar", "length": 1, "colcomm": "撤消标识"},
    {"colname": "DEVTYP", "type": "varchar", "length": 1, "colcomm": "设备类型"},
    {"colname": "RECSTS", "type": "varchar", "length": 1, "colcomm": "记录状态"},
    {"colname": "RECTIM", "type": "varchar", "length": 6, "colcomm": "状态更改时间"},
    {"colname": "DLGMRK", "type": "varchar", "length": 24, "colcomm": "备注"}
]



# 在文件顶部设置日志（或在 main 中设置）
# logging.basicConfig(
#     level=logging.INFO,
#     format='%(asctime)s | %(levelname)-8s | %(funcName)s:%(lineno)d - %(message)s',
#     handlers=[
#         logging.FileHandler("debug.log", encoding='utf-8'),  # 写入文件
#         logging.StreamHandler()  # 同时输出到控制台
#     ]
# )

def out_config_actvhh():
    # 模板只编译一次，省 CPU
    tmpl = Template('''\
<?xml version="1.0" encoding="utf-8" ?>
<config>
    <tabname>{{ tab_name }}</tabname>
    <user>{{ user }}</user>
    <tabcomm>{{ tab_comm }}</tabcomm>
    <columnsamount>{{ cols_amt }}</columnsamount>
    <recordsamount>{{ recs_amt }}</recordsamount>
    <seperator>{{ sep }}</seperator>
    <partinfo>{{ part_info }}</partinfo>
    <system>
        <sysname>{{ sys_name }}</sysname>
        <sysversion>{{ sys_ver }}</sysversion>
        <syscomm>{{ sys_comm }}</syscomm>
        <infcenterid>{{ infc_id }}</infcenterid>
        <regcod></regcod>
    </system>
    <backup>
        <time>{{ back_time }}</time>
        <type>backbackup</type>
    </backup>
    <export>
        <daterangecols>{{ daterange }}</daterangecols>
    </export>
    <columns>
    {%for col in actvhh_columns%}
        <column>
            <colname>{{ col.colname }}</colname>
            <type>{{ col.type }}</type>
            <length>{{ col.length }}</length>
            <colcomm>{{ col.colcomm }}</colcomm>
        </column>
                        {%-endfor%}
    </columns>
</config>''')
    for index in range(4):
        if not config_paths[index]:  # 空路径跳过
            continue
        print(f'获取输出目录：{config_paths[index]}')
        out_path = pathlib.Path(config_paths[index])
        out_path.mkdir(parents=True, exist_ok=True)
        print(f'输出out_path={out_path}')
        # 文件名
        configname = f"{actvhh_config["tab_name"]}-{user}-{back_times[index]}-A-b.cfg.xml"
        out_file = out_path / configname
        print(f'输出out_file={out_file}')
        # 渲染模板：把当前 index 传进去
        xml_content = tmpl.render(
            tab_name=actvhh_config["tab_name"],
            user=user,
            tab_comm=actvhh_config["tab_comm"],
            cols_amt=actvhh_config["cols_amt"],
            recs_amt=recs_amts[index],  # 整个列表传进去
            sep=sep,
            part_info=part_infos[index],
            sys_name=sys_name,
            sys_ver=sys_ver,
            sys_comm=sys_comm,
            infc_id=infc_id,
            back_time=back_times[index],
            daterange=actvhh_config["daterange"],
            actvhh_columns=actvhh_columns,
        )
    #     写入配置文件
        # 写文件
        with open(out_file, 'w', encoding='utf-8') as f:
            f.write(xml_content)
    return

def out_config_cdtdlg():
    # 模板只编译一次，省 CPU
    tmpl = Template('''\
<?xml version="1.0" encoding="utf-8" ?>
<config>
    <tabname>{{ tab_name }}</tabname>
    <user>{{ user }}</user>
    <tabcomm>{{ tab_comm }}</tabcomm>
    <columnsamount>{{ cols_amt }}</columnsamount>
    <recordsamount>{{ recs_amt }}</recordsamount>
    <seperator>{{ sep }}</seperator>
    <partinfo>{{ part_info }}</partinfo>
    <system>
        <sysname>{{ sys_name }}</sysname>
        <sysversion>{{ sys_ver }}</sysversion>
        <syscomm>{{ sys_comm }}</syscomm>
        <infcenterid>{{ infc_id }}</infcenterid>
        <regcod></regcod>
    </system>
    <backup>
        <time>{{ back_time }}</time>
        <type>backbackup</type>
    </backup>
    <export>
        <daterangecols>{{ daterange }}</daterangecols>
    </export>
    <columns>
    {%for col in cdtdlg_columns%}
        <column>
            <colname>{{ col.colname }}</colname>
            <type>{{ col.type }}</type>
            <length>{{ col.length }}</length>
            <colcomm>{{ col.colcomm }}</colcomm>
        </column>
                        {%-endfor%}
    </columns>
</config>''')
    for index in range(4):
        if not config_paths[index]:  # 空路径跳过
            continue
        out_path = pathlib.Path(config_paths[index])
        out_path.mkdir(parents=True, exist_ok=True)
        # 文件名
        configname = f"{cdtdlg_config["tab_name"]}-{user}-{back_times[index]}-A-b.cfg.xml"
        out_file = out_path / configname
        # 渲染模板：把当前 index 传进去
        xml_content = tmpl.render(
            tab_name=cdtdlg_config["tab_name"],
            user=user,
            tab_comm=cdtdlg_config["tab_comm"],
            cols_amt=cdtdlg_config["cols_amt"],
            recs_amt=recs_amts[index],  # 整个列表传进去
            sep=sep,
            part_info=part_infos[index],
            sys_name=sys_name,
            sys_ver=sys_ver,
            sys_comm=sys_comm,
            infc_id=infc_id,
            back_time=back_times[index],
            daterange=cdtdlg_config["daterange"],
            cdtdlg_columns=cdtdlg_columns,
        )
    #     写入配置文件
        # 写文件
        with open(out_file, 'w', encoding='utf-8') as f:
            f.write(xml_content)
    return




def out_config_rettxn():
    # 模板只编译一次，省 CPU
    tmpl = Template('''\
<?xml version="1.0" encoding="utf-8" ?>
<config>
    <tabname>{{ tab_name }}</tabname>
    <user>{{ user }}</user>
    <tabcomm>{{ tab_comm }}</tabcomm>
    <columnsamount>{{ cols_amt }}</columnsamount>
    <recordsamount>{{ recs_amt }}</recordsamount>
    <seperator>{{ sep }}</seperator>
    <partinfo>{{ part_info }}</partinfo>
    <system>
        <sysname>{{ sys_name }}</sysname>
        <sysversion>{{ sys_ver }}</sysversion>
        <syscomm>{{ sys_comm }}</syscomm>
        <infcenterid>{{ infc_id }}</infcenterid>
        <regcod></regcod>
    </system>
    <backup>
        <time>{{ back_time }}</time>
        <type>backbackup</type>
    </backup>
    <export>
        <daterangecols>{{ daterange }}</daterangecols>
    </export>
    <columns>
    {%for col in rettxn_columns%}
        <column>
            <colname>{{ col.colname }}</colname>
            <type>{{ col.type }}</type>
            <length>{{ col.length }}</length>
            <colcomm>{{ col.colcomm }}</colcomm>
        </column>
                        {%-endfor%}
    </columns>
</config>''')
    for index in range(4):
        # 创建目录
        # out_path = pathlib.Path(config_paths[index])
        # out_path.mkdir(parents=True, exist_ok=True)
        # print(f'配置文件输出目录：{out_path}')
        #
        # configname = tab_name +'-'+user +back_times[index]+'-A-b.cfg.xml'
        if not config_paths[index]:  # 空路径跳过
            continue
        out_path = pathlib.Path(config_paths[index])
        out_path.mkdir(parents=True, exist_ok=True)
        # 文件名
        configname = f"{rettxn_config["tab_name"]}-{user}-{back_times[index]}-A-b.cfg.xml"
        out_file = out_path / configname
        # 渲染模板：把当前 index 传进去
        xml_content = tmpl.render(
            tab_name=rettxn_config["tab_name"],
            user=user,
            tab_comm=rettxn_config["tab_comm"],
            cols_amt=rettxn_config["cols_amt"],
            recs_amt=recs_amts[index],  # 整个列表传进去
            sep=sep,
            part_info=part_infos[index],
            sys_name=sys_name,
            sys_ver=sys_ver,
            sys_comm=sys_comm,
            infc_id=infc_id,
            back_time=back_times[index],
            daterange=rettxn_config["daterange"]    ,
            rettxn_columns=rettxn_columns,
        )
    #     写入配置文件
        # 写文件
        with open(out_file, 'w', encoding='utf-8') as f:
            f.write(xml_content)
    return

# 扫描输入路径下的所有文件
def scan_files_non_recursive(root_path , out_path):
    global config_paths, back_times, part_infos, recs_amts
    root = pathlib.Path(root_path)
    if not root.exists() or not root.is_dir():
        raise ValueError("无效目录")
    files = [f for f in root.iterdir() if f.is_file()]
    if not files:
        print("目录中没有文件，跳过处理。")
        return
    for file in files:
        deduplicate_by_field(file ,out_path)
        file_name_upper = file.name.upper()
        if "ACTVHH" in file_name_upper:
            out_config_actvhh()
        if "RETTXN" in file_name_upper:
            out_config_rettxn()
        if "CDTDLG" in file_name_upper:
            out_config_cdtdlg()
        # 重置！否则会污染后续文件处理
        config_paths = [''] * 4
        back_times = [''] * 4
        part_infos = [0] * 4
        recs_amts = [0] * 4
    return




def deduplicate_by_field(input_file , output_base ):
    print(f"正在处理文件：{input_file}")
    print(f"输出目录：{output_base}" )
    with open(input_file, 'r', encoding='utf8') as f:
        # 数据记录二次去重
        seen = set()
        unique_records = []
        records = f.readlines()
        for record in records:
            stripped = record.strip()
            # 去掉数据记录前的序号
            no_seq = stripped.split('|', 1)[1].strip()
            if no_seq not in seen:
                unique_records.append(record)  # 保留原始行（含\n）
                seen.add(no_seq)
            else :
                print("捕获到重复数据：" + record)
        sx_records = []  # 机构号  1006 - 1299
        nm_records = []  # 机构号  1301 - 1599
        jl_records = []  # 机构号  2009 - 2299
        hl_records = []  # 机构号  2301 - 2557
        yc_records = [] #异常数据
    #     根据机构号拆分数据
    #     flag = 1
        for record in unique_records:
            # flag = flag + 1
            if record.strip() == '':
                continue
            target = record.strip().split('|')
            #判断ACTVHH 或者 RETTXN
            if len(target) == ACTVHH_LEN:
                # if flag == 10 : print(f'获取ACTVHH 记录长度{len(target)}')
                try:
                    org_id = int(target[ACTVHH_INDEX])
                    # if flag == 10: print(f'获取ACTVHH 记录org_id{org_id}')
                except (ValueError, IndexError):
                    parts = record.split('|')
                    parts[0] = str(len(yc_records) + 1)
                    yc_records.append('|'.join(parts))
                    # yc_records.append(record)
                    continue
                if sx_org_begin <= org_id <= sx_org_end:
                    # if flag < 1000: print('山西：',org_id)
                    # parts = record.split('|')
                    # parts[0] = str(len(sx_records)+1)
                    # sx_records.append('|'.join(parts))
                    sx_records.append(record)
                #修改序号
                elif nm_org_begin <= org_id <= nm_org_end:
                    # if flag < 1000: print('内蒙古：',org_id)
                    nm_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(nm_records) + 1)
                    # nm_records.append('|'.join(parts))
                elif jl_org_begin <= org_id <= jl_org_end:
                    # if flag < 1000: print('吉林：',org_id)
                    jl_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(jl_records) + 1)
                    # jl_records.append('|'.join(parts))
                elif hl_org_begin <= org_id <= hl_org_end:
                    # if flag < 10000: print("黑龙江：",org_id)
                    hl_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(hl_records) + 1)
                    # hl_records.append('|'.join(parts))
                else:
                    yc_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(yc_records) + 1)
                    # yc_records.append('|'.join(parts))
            # RETTXN 逻辑处理 需要字段合并
            # if flag < 100 :
            #     # print(f'RETTXN 长度为：{len(target)}')
            #     print(record)
            #     print(target)
            if len(target) == RETTXN_LEN:
                # try:
                #     org_id = int(target[RETTXN_INDEX])
                #     # print(f'org_id: {org_id}')
                # except (ValueError, IndexError):
                #     yc_records.append(record)
                #     continue
                try:
                    # raw_value = target[RETTXN_INDEX] if RETTXN_INDEX < len(target) else None
                    # if raw_value is None:
                    #     raw_value = target[RETTXN_INDEX_B]
                    # org_id = int(raw_value)
                    raw_value = target[RETTXN_INDEX].strip()
                    if raw_value == "":
                        raw_value = target[RETTXN_INDEX_B]
                    org_id = int(raw_value)
                except (ValueError, IndexError):
                    # yc_records.append(record)
                    parts = record.split('|')
                    parts[0] = str(len(yc_records) + 1)
                    yc_records.append('|'.join(parts))
                    continue
                if sx_org_begin <= org_id <= sx_org_end:
                    sx_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(sx_records) + 1)
                    # sx_records.append('|'.join(parts))
                elif nm_org_begin <= org_id <= nm_org_end:
                    nm_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(nm_records) + 1)
                    # nm_records.append('|'.join(parts))
                elif jl_org_begin <= org_id <= jl_org_end:
                    jl_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(jl_records) + 1)
                    # jl_records.append('|'.join(parts))
                elif hl_org_begin <= org_id <= hl_org_end:
                    hl_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(hl_records) + 1)
                    # hl_records.append('|'.join(parts))
                else:
                    yc_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(yc_records) + 1)
                    # yc_records.append('|'.join(parts))
            if len(target) == CDTDLG_LEN:
                # try:
                #     org_id = int(target[CDTDLG_INDEX]) or int(target[CDTDLG_INDEX_B])
                #     # print(f'org_id: {org_id}')
                try:
                    # raw_value = target[CDTDLG_INDEX] if CDTDLG_INDEX < len(target) else None
                    # if raw_value is None:
                    #     raw_value = target[CDTDLG_INDEX_B]
                    raw_value = target[CDTDLG_INDEX].strip()
                    # 修复raw_value是字符串的情况
                    # if raw_value == "":
                    #     raw_value = target[CDTDLG_INDEX_B]
                    # org_id = int(raw_value)
                    # 如果为空 **或者** 不是纯数字，就退回到备用字段
                    if raw_value == "" or not raw_value.isdigit():
                        raw_value = target[CDTDLG_INDEX_B].strip()
                        # 备用字段也可能不是数字，再检查一次
                        if not raw_value.isdigit():
                            # 两条路都走不通，记入异常数据
                            raise ValueError("机构号非数字")
                    org_id = int(raw_value)
                except (ValueError, IndexError):
                    yc_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(yc_records) + 1)
                    # yc_records.append('|'.join(parts))
                    continue
                if sx_org_begin <= org_id <= sx_org_end:
                    sx_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(sx_records) + 1)
                    # sx_records.append('|'.join(parts))
                elif nm_org_begin <= org_id <= nm_org_end:
                    nm_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(nm_records) + 1)
                    # nm_records.append('|'.join(parts))
                elif jl_org_begin <= org_id <= jl_org_end:
                    jl_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(jl_records) + 1)
                    # jl_records.append('|'.join(parts))
                elif hl_org_begin <= org_id <= hl_org_end:
                    hl_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(hl_records) + 1)
                    # hl_records.append('|'.join(parts))
                else:
                    yc_records.append(record)
                    # parts = record.split('|')
                    # parts[0] = str(len(yc_records) + 1)
                    # yc_records.append('|'.join(parts))
    # 创建目录/目标文件 山西
        if  sx_records:
            # 创建目录
            path = pathlib.Path(output_base) / out_path_sx
            # 提取文件名
            input_file_name = pathlib.PurePath(input_file).stem
            path = path / input_file_name.split('-')[2]
            # cut = input_file_name.rfind('-', 0, input_file_name.rfind('-'))
            # path = path / input_file_name[:cut]
            print(f'山西path路径{path}')
            # path = pathlib.Path(output_base/out_path_sx)
            path.mkdir(parents=True, exist_ok=True)
    #         获取配置文件参数
    #         获取备份路径
            config_paths[0] = path
            # 获取备份时间
            # print(input_file_name.split('-'))
            back_times[0] = input_file_name.split('-')[2]

    #       获取文件条目数
            recs_amts[0] = recs_amts[0]+ len(sx_records)
            #         写入文件
            processing_path[0] = path
            # 将列表转化为队列
            sx_records = deque(sx_records)

            CHUNK = 1_700_001

            filename_base = input_file_name
            if 'RETTXN' in filename_base.upper():
                CHUNK = 1_300_000
            while sx_records:  # 只要还有数据
                part_infos[0] += 1  # 文件计数器
                # 设置文件名
                input_file_name = f'{filename_base}.p{part_infos[0]:03d}'
                with open(path / input_file_name, 'w', encoding='utf8') as f:
                    f.write(begin)  # 文件头
                    # 安全地取 1 800 000 条（不足就全取）
                    lines = []
                    # 实际弹出最多 CHUNK 条记录
                    for i in range(min(CHUNK, len(sx_records))):
                        record = sx_records.popleft()  # 关键：真正移除
                        parts = record.split('|')
                        parts[0] = str(i + 1)  # 序号从 1 开始
                        lines.append('|'.join(parts))
                    f.writelines(lines)
                    f.write(end)  # 文件尾
                    print(f'成功拆分山西ODMS数据，已写入 {input_file_name}')

        if  nm_records:
            # 创建目录
            # path = pathlib.Path(output_base/out_path_nm)
            path = pathlib.Path(output_base) / out_path_nm
            # 提取文件名
            input_file_name = pathlib.PurePath(input_file).stem
            path = path / input_file_name.split('-')[2]
            # cut = input_file_name.rfind('-', 0, input_file_name.rfind('-'))
            # path = path / input_file_name[:cut]
            print(f'内蒙古path路径{path}')
            path.mkdir(parents=True, exist_ok=True)
            #         获取配置文件参数
            #         获取备份路径
            config_paths[1] = path
            # 获取备份时间
            back_times[1] = input_file_name.split('-')[2]
            # #       获取备份文件个数
            # part_infos[1] = part_infos[1] + 1
            #       获取文件条目数
            recs_amts[1] = recs_amts[1] + len(nm_records)
    #         写入文件
            processing_path[1] = path

            # with open(path / input_file_name , 'w', encoding='utf8') as f:
            #     # print(f'输出路径：{path / input_file_name}')
            #     f.write(begin)
            #     f.writelines(nm_records)
            #     f.write(end)
            #     # print(f'成功拆分内蒙ODMS数据并且成功写入{len(nm_records)}条数据')
            # 将列表转化为队列
            nm_records = deque(nm_records)
            CHUNK = 1_700_001
            filename_base = input_file_name
            if 'RETTXN' in filename_base.upper():
                CHUNK = 1_300_000
            while nm_records:  # 只要还有数据
                part_infos[1] += 1  # 文件计数器
                # 设置文件名
                input_file_name = f'{filename_base}.p{part_infos[1]:03d}'
                with open(path / input_file_name, 'w', encoding='utf8') as f:
                    f.write(begin)  # 文件头
                    # 安全地取 1 800 000 条（不足就全取）
                    lines = []
                    # 实际弹出最多 CHUNK 条记录
                    for i in range(min(CHUNK, len(nm_records))):
                        record = nm_records.popleft()  # 关键：真正移除
                        parts = record.split('|')
                        parts[0] = str(i + 1)  # 序号从 1 开始
                        lines.append('|'.join(parts))
                    f.writelines(lines)
                    f.write(end)  # 文件尾
                    print(f'成功拆分内蒙ODMS数据，已写入 {input_file_name}')
        if  jl_records:
            # 创建目录
            # path = pathlib.Path(output_base/out_path_jl)
            path = pathlib.Path(output_base) / out_path_jl
            # 提取文件名
            input_file_name = pathlib.PurePath(input_file).stem
            path = path / input_file_name.split('-')[2]
            # path = path / input_file_name[:input_file_name.rfind('-')]
            # cut = input_file_name.rfind('-', 0, input_file_name.rfind('-'))
            # path = path / input_file_name[:cut]
            # print(f'吉林path路径{path}')
            path.mkdir(parents=True, exist_ok=True)
            #         获取配置文件参数
            #         获取备份路径
            config_paths[2] = path
            # 获取备份时间
            back_times[2] = input_file_name.split('-')[2]
            # #       获取备份文件个数
            # part_infos[2] = part_infos[2] + 1
            #       获取文件条目数
            recs_amts[2] = recs_amts[2] + len(jl_records)
    #         写入文件
            processing_path[2] = path
            # 将列表转化为队列
            jl_records = deque(jl_records)
            CHUNK = 1_700_001
            filename_base = input_file_name
            if 'RETTXN' in filename_base.upper():
                CHUNK = 1_300_000
            while jl_records:  # 只要还有数据
                part_infos[2] += 1  # 文件计数器
                # 设置文件名
                input_file_name = f'{filename_base}.p{part_infos[2]:03d}'
                with open(path / input_file_name, 'w', encoding='utf8') as f:

                    f.write(begin)  # 文件头
                    # 安全地取 1 800 000 条（不足就全取）
                    lines = []
                    # 实际弹出最多 CHUNK 条记录
                    for i in range(min(CHUNK, len(jl_records))):
                        record = jl_records.popleft()  # 关键：真正移除
                        parts = record.split('|')
                        parts[0] = str(i + 1)  # 序号从 1 开始
                        lines.append('|'.join(parts))
                    f.writelines(lines)
                    f.write(end)  # 文件尾
                    print(f'成功拆分吉林ODMS数据，已写入 {input_file_name}')
        if hl_records:
            # 创建目录
            # path = pathlib.Path(output_base/out_path_hl)
            path = pathlib.Path(output_base) / out_path_hl
            # 提取文件名
            input_file_name = pathlib.PurePath(input_file).stem
            path = path / input_file_name.split('-')[2]
            # path = path / input_file_name[:input_file_name.rfind('-')]
            # cut = input_file_name.rfind('-', 0, input_file_name.rfind('-'))
            # path = path / input_file_name[:cut]
            # print(f'黑龙江path路径{path}')
            path.mkdir(parents=True, exist_ok=True)
            #         获取配置文件参数
            #         获取备份路径
            config_paths[3] = path
            # 获取备份时间
            back_times[3] = input_file_name.split('-')[2]
            # #       获取备份文件个数
            # part_infos[3] = part_infos[3] + 1
            #       获取文件条目数
            recs_amts[3] = recs_amts[3] + len(hl_records)
    #         写入文件
            processing_path[3] = path
            # with open(path / input_file_name , 'w', encoding='utf8') as f:
            #     # print(f'输出路径：{path / input_file_name}')
            #     f.write(begin)
            #     f.writelines(hl_records)
            #     f.write(end)
            #     print(f'成功拆分黑龙江ODMS数据并且成功写入{len(hl_records)}条数据')
            # 将列表转化为队列
            hl_records = deque(hl_records)
            CHUNK = 1_700_001
            filename_base = input_file_name
            if 'RETTXN' in filename_base.upper():
                CHUNK = 1_300_000
            while hl_records:  # 只要还有数据
                part_infos[3] += 1  # 文件计数器
                # 设置文件名
                input_file_name = f'{filename_base}.p{part_infos[3]:03d}'
                with open(path / input_file_name, 'w', encoding='utf8') as f:
                    f.write(begin)  # 文件头
                    # 安全地取 1 800 000 条（不足就全取）
                    lines = []
                    # 实际弹出最多 CHUNK 条记录
                    for i in range(min(CHUNK, len(hl_records))):
                        record = hl_records.popleft()  # 关键：真正移除
                        parts = record.split('|')
                        parts[0] = str(i + 1)  # 序号从 1 开始
                        lines.append('|'.join(parts))
                    f.writelines(lines)
                    f.write(end)  # 文件尾
                    print(f'成功拆分黑龙江ODMS数据，已写入 {input_file_name}')
        if yc_records:
            # 创建目录
            # path = pathlib.Path(output_base/out_path_e)
            path = pathlib.Path(output_base) / out_path_e
            # 提取文件名
            input_file_name = pathlib.PurePath(input_file).name
            path = path / input_file_name.split('-')[2]
            # path = path / input_file_name[:input_file_name.rfind('-')]
            # cut = input_file_name.rfind('-', 0, input_file_name.rfind('-'))
            # path = path / input_file_name[:cut]
            path.mkdir(parents=True, exist_ok=True)
    #         写入文件
            output_name = f"{input_file.stem}-e{input_file.suffix}"
            with open(path / output_name , 'w', encoding='utf8') as f:
                # print(f'输出路径：{path / input_file}')
                f.write(begin)
                f.writelines(yc_records)
                f.write(end)
                print(f'成功获取异常数据并且成功写入{len(yc_records)}条数据')
    return



# 最终数据去重
def data_deduplication():
    for index in range(0 , len(config_paths)):
        print(f"获取输出文件路径: {config_paths[index]}")
    """========设计配置文件========"""
    global part_infos , recs_amts
    # 输出个数
    part_infos_t = [0] * 4
    # 文件条数
    recs_amts_t = [0] * 4

    # 设计文件输出路径
    for index_1 in range(len(config_paths)):
        # 空路径跳过
        if not config_paths[index_1]: continue
        # print(f"获取输出文件路径_2: {type(config_paths[index_1])}")
        out_path = config_paths[index_1]
        out_path.mkdir(parents=True, exist_ok=True)
        # 设计输出文件名
        filename_head = out_path.name
        # print(f'捕捉到需要处理的路径 out ：{out_path}')
        print(f'获取输出文件头：{filename_head}')
        # continue
        # 件记录缓存
        # # 扫描当前目录下所有文件
        # 设计文件记录去重
        unique = set()
        for file in out_path.iterdir():
            if file.is_file():
                print(f'获取{file.name}记录数据')
                with open(file, "r", encoding='utf8') as f:
                    lines = f.readlines()
                    for index_2 in range(1, len(lines) - 1):
                        if lines[index_2] in unique:
                            # 捕获到重复数据
                            print("捕获到重复数据: " + lines[index_2])
                        else:
                            unique.add(lines[index_2])
            if remove : unlink(file)
        # 统计所有文件记录条数
        recs_amts_t[index_1] = len(unique)
        unique = deque(unique)  # 转为双端队列，支持高效 popleft
        part = 0
        BATCH_SIZE = 1_600_000
        # 配置文件参数获取 =======================

        while unique:
            part += 1
            # 统计生成文件个数
            part_infos_t[index_1] = part
            filename = f"{filename_head}-A-b.p{part:03d}"  # 格式化为 p001, p010, p100 等
            with open(out_path / filename, "w", encoding='utf-8') as f:
                print(f'文件输出路径：{out_path / filename}')
                f.write('|||BEGIN|||\n')
                count = 0
                while unique and count < BATCH_SIZE:
                    record = unique.popleft()  # 安全弹出
                    count += 1
                    parts = record.split('|')
                    parts[0] = str(count)  # 重写序号（注意：这是文件内序号，不是全局）
                    f.write('|'.join(parts))
                    # 注意：如果原记录末尾没有换行符，可能需要加 \n
                    # f.write('|'.join(parts) + '\n')
                f.write('|||END|||\n')
    recs_amts= recs_amts_t
    part_infos = part_infos_t
    return




def main():
    # global remove
    # 获取输入目录
    input_path = input("输入处理文件目录ACTVHH/RETTXN: ").strip()
    out_path = input("输入处理文件的输出目录：").strip()
    # flag = input("是否保留多余数据：(y/n): ").strip()
    # if flag == 'y':
    #     remove = False

    logging.info(f"开始处理文件: {input_path}")
    # 初始化全局变量
    input_path = unicodedata.normalize('NFC', input_path).strip('\u202a\u202b\u200e\u200f')

    out_path = unicodedata.normalize('NFC', out_path).strip('\u202a\u202b\u200e\u200f')
    scan_files_non_recursive(input_path , out_path)
    # 处理最终文件
    # data_deduplication()
    # 生成配置文件


    return

if __name__ == '__main__':

    main()

